﻿using System;
using System.Collections;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using System.Windows.Forms;
using System.IO;
using static System.Windows.Forms.VisualStyles.VisualStyleElement.ProgressBar;
using System.Security.Policy;

namespace homework8_2
{
    public partial class Form1 : Form
    {
            public Form1()
        {
            InitializeComponent();
        }
        
        private void button1_Click(object sender, EventArgs e)
        {
            listBox1.Items.Clear();
            listBox1.Items.Add("开始加载。。。");
            Crawler crawler = new Crawler();
            string startUrl = textBox1.Text;
            crawler.urls.Add(startUrl, false);
            new Thread(crawler.Crawl).Start();
            listBox1.Items.Add("加载完成");
        }
        
    }
    public class Crawler : Form1
    {
        public Hashtable urls = new Hashtable();
        public int count = 0;

        public void Crawl()
        {
            Task.Run(() =>
            {
                while (true)
                {
                    string current = null;
                    foreach (string url in urls.Keys)
                    {
                        if ((bool)urls[url])
                        {
                            continue;
                        }
                        current = url;
                        break; // 假设你只想处理一个 URL，然后再次循环  
                    }

                    if (current == null || count > 15)
                        break;

                    // 使用 Invoke 确保在 UI 线程上更新 ListBox  
                    if (listBox1.InvokeRequired)
                    {
                        listBox1.Invoke(new Action(() =>
                        {
                            listBox1.Items.Add(current);
                        }));
                    }
                    else
                    {
                        listBox1.Items.Add(current); // 如果已经在 UI 线程上，则直接添加  
                    }
                    string html = DownLoad(current);
                    urls[current] = true;
                    count++;
                    Parse(html, current);
                }
            });
        }
        //DownLoad方法：使用WebClient下载指定URL的HTML内容。将HTML内容写入以count命名的文件。返回HTML内容。
        public string DownLoad(string url)
        {
            try
            {
                WebClient client = new WebClient();
                client.Encoding = Encoding.UTF8;
                string html = client.DownloadString(url);
                string fileName = count.ToString();      //文件名定义为0,1,2,3,4
                File.WriteAllText(fileName, html, Encoding.UTF8);  //将html代码写入文件
                return html;


            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.Message);
                return "";
            }
        }
        public void Parse(string html, string oldUrl)
        {
            //只有当爬取的是HTML文本时，才解析并爬取下一级URL
            string strRef = @"(href|HREF)[ ]*=[ ]*[""'](http|https)[^""'#>]+..html.*?[""']";

            MatchCollection matchCollection = new Regex(strRef).Matches(html);
            foreach (Match match in matchCollection)
            {
                var url = match.Value.Substring(match.Value.IndexOf('=') + 1).Trim('"', '\"', '#', ' ', '>');
                if (url.Length == 0)
                    continue;

                //相对地址转成绝对地址进行爬取
                Uri baseUri = new Uri(oldUrl);
                Uri absoluteUri = new Uri(baseUri, url);

                //只爬取起始网站上的网页
                if (url.Contains("https://www.cnblogs.com"))
                {
                    if (urls[absoluteUri.ToString()] == null)
                        urls[absoluteUri.ToString()] = false;
                }
            }
        }
    }

}
